3 * Job to update link tables for pages
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
23 use MediaWiki\MediaWikiServices
;
24 use MediaWiki\Revision\RevisionRecord
;
27 * Job to update link tables for pages
29 * This job comes in a few variants:
30 * - a) Recursive jobs to update links for backlink pages for a given title.
31 * These jobs have (recursive:true,table:<table>) set.
32 * - b) Jobs to update links for a set of pages (the job title is ignored).
33 * These jobs have (pages:(<page ID>:(<namespace>,<title>),...) set.
34 * - c) Jobs to update links for a single page (the job title)
35 * These jobs need no extra fields set.
39 class RefreshLinksJob
extends Job
{
40 /** @var float Cache parser output when it takes this long to render */
41 const PARSE_THRESHOLD_SEC
= 1.0;
42 /** @var int Lag safety margin when comparing root job times to last-refresh times */
43 const CLOCK_FUDGE
= 10;
44 /** @var int How many seconds to wait for replica DBs to catch up */
45 const LAG_WAIT_TIMEOUT
= 15;
47 function __construct( Title
$title, array $params ) {
48 parent
::__construct( 'refreshLinks', $title, $params );
49 // Avoid the overhead of de-duplication when it would be pointless
50 $this->removeDuplicates
= (
51 // Ranges rarely will line up
52 !isset( $params['range'] ) &&
53 // Multiple pages per job make matches unlikely
54 !( isset( $params['pages'] ) && count( $params['pages'] ) != 1 )
56 $this->params +
= [ 'causeAction' => 'unknown', 'causeAgent' => 'unknown' ];
61 * @param array $params
62 * @return RefreshLinksJob
64 public static function newPrioritized( Title
$title, array $params ) {
65 $job = new self( $title, $params );
66 $job->command
= 'refreshLinksPrioritized';
73 * @param array $params
74 * @return RefreshLinksJob
76 public static function newDynamic( Title
$title, array $params ) {
77 $job = new self( $title, $params );
78 $job->command
= 'refreshLinksDynamic';
84 global $wgUpdateRowsPerJob;
87 // Job to update all (or a range of) backlink pages for a page
88 if ( !empty( $this->params
['recursive'] ) ) {
89 // When the base job branches, wait for the replica DBs to catch up to the master.
90 // From then on, we know that any template changes at the time the base job was
91 // enqueued will be reflected in backlink page parses when the leaf jobs run.
92 if ( !isset( $this->params
['range'] ) ) {
93 $lbFactory = MediaWikiServices
::getInstance()->getDBLoadBalancerFactory();
94 if ( !$lbFactory->waitForReplication( [
95 'domain' => $lbFactory->getLocalDomainID(),
96 'timeout' => self
::LAG_WAIT_TIMEOUT
97 ] ) ) { // only try so hard
98 $stats = MediaWikiServices
::getInstance()->getStatsdDataFactory();
99 $stats->increment( 'refreshlinks.lag_wait_failed' );
102 // Carry over information for de-duplication
103 $extraParams = $this->getRootJobParams();
104 $extraParams['triggeredRecursive'] = true;
105 // Carry over cause information for logging
106 $extraParams['causeAction'] = $this->params
['causeAction'];
107 $extraParams['causeAgent'] = $this->params
['causeAgent'];
108 // Convert this into no more than $wgUpdateRowsPerJob RefreshLinks per-title
109 // jobs and possibly a recursive RefreshLinks job for the rest of the backlinks
110 $jobs = BacklinkJobUtils
::partitionBacklinkJob(
114 [ 'params' => $extraParams ]
116 JobQueueGroup
::singleton()->push( $jobs );
117 // Job to update link tables for a set of titles
118 } elseif ( isset( $this->params
['pages'] ) ) {
119 foreach ( $this->params
['pages'] as list( $ns, $dbKey ) ) {
120 $title = Title
::makeTitleSafe( $ns, $dbKey );
122 $this->runForTitle( $title );
125 $this->setLastError( "Invalid title ($ns,$dbKey)." );
128 // Job to update link tables for a given title
130 $this->runForTitle( $this->title
);
137 * @param Title $title
140 protected function runForTitle( Title
$title ) {
141 $services = MediaWikiServices
::getInstance();
142 $stats = $services->getStatsdDataFactory();
143 $lbFactory = $services->getDBLoadBalancerFactory();
144 $revisionStore = $services->getRevisionStore();
145 $renderer = $services->getRevisionRenderer();
146 $ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__
);
148 $page = WikiPage
::factory( $title );
149 $page->loadPageData( WikiPage
::READ_LATEST
);
151 // Serialize links updates by page ID so they see each others' changes
152 $dbw = $lbFactory->getMainLB()->getConnection( DB_MASTER
);
153 /** @noinspection PhpUnusedLocalVariableInspection */
154 $scopedLock = LinksUpdate
::acquirePageLock( $dbw, $page->getId(), 'job' );
155 if ( $scopedLock === null ) {
156 // Another job is already updating the page, likely for an older revision (T170596).
157 $this->setLastError( 'LinksUpdate already running for this page, try again later.' );
160 // Get the latest ID *after* acquirePageLock() flushed the transaction.
161 // This is used to detect edits/moves after loadPageData() but before the scope lock.
162 // The works around the chicken/egg problem of determining the scope lock key.
163 $latest = $title->getLatestRevID( Title
::GAID_FOR_UPDATE
);
165 if ( !empty( $this->params
['triggeringRevisionId'] ) ) {
166 // Fetch the specified revision; lockAndGetLatest() below detects if the page
167 // was edited since and aborts in order to avoid corrupting the link tables
168 $revision = $revisionStore->getRevisionById(
169 (int)$this->params
['triggeringRevisionId'],
170 Revision
::READ_LATEST
173 // Fetch current revision; READ_LATEST reduces lockAndGetLatest() check failures
174 $revision = $revisionStore->getRevisionByTitle( $title, 0, Revision
::READ_LATEST
);
178 $stats->increment( 'refreshlinks.rev_not_found' );
179 $this->setLastError( "Revision not found for {$title->getPrefixedDBkey()}" );
180 return false; // just deleted?
181 } elseif ( $revision->getId() != $latest ||
$revision->getPageId() !== $page->getId() ) {
182 // Do not clobber over newer updates with older ones. If all jobs where FIFO and
183 // serialized, it would be OK to update links based on older revisions since it
184 // would eventually get to the latest. Since that is not the case (by design),
185 // only update the link tables to a state matching the current revision's output.
186 $stats->increment( 'refreshlinks.rev_not_current' );
187 $this->setLastError( "Revision {$revision->getId()} is not current" );
191 $parserOutput = false;
192 $parserOptions = $page->makeParserOptions( 'canonical' );
193 // If page_touched changed after this root job, then it is likely that
194 // any views of the pages already resulted in re-parses which are now in
195 // cache. The cache can be reused to avoid expensive parsing in some cases.
196 if ( isset( $this->params
['rootJobTimestamp'] ) ) {
197 $opportunistic = !empty( $this->params
['isOpportunistic'] );
199 $skewedTimestamp = $this->params
['rootJobTimestamp'];
200 if ( $opportunistic ) {
201 // Neither clock skew nor DB snapshot/replica DB lag matter much for such
202 // updates; focus on reusing the (often recently updated) cache
204 // For transclusion updates, the template changes must be reflected
205 $skewedTimestamp = wfTimestamp( TS_MW
,
206 wfTimestamp( TS_UNIX
, $skewedTimestamp ) + self
::CLOCK_FUDGE
210 if ( $page->getLinksTimestamp() > $skewedTimestamp ) {
211 // Something already updated the backlinks since this job was made
212 $stats->increment( 'refreshlinks.update_skipped' );
216 if ( $page->getTouched() >= $this->params
['rootJobTimestamp'] ||
$opportunistic ) {
217 // Cache is suspected to be up-to-date. As long as the cache rev ID matches
218 // and it reflects the job's triggering change, then it is usable.
219 $parserOutput = $services->getParserCache()->getDirty( $page, $parserOptions );
221 ||
$parserOutput->getCacheRevisionId() != $revision->getId()
222 ||
$parserOutput->getCacheTime() < $skewedTimestamp
224 $parserOutput = false; // too stale
229 // Fetch the current revision and parse it if necessary...
230 if ( $parserOutput ) {
231 $stats->increment( 'refreshlinks.parser_cached' );
233 $start = microtime( true );
235 $checkCache = $page->shouldCheckParserCache( $parserOptions, $revision->getId() );
237 // Revision ID must be passed to the parser output to get revision variables correct
238 $renderedRevision = $renderer->getRenderedRevision(
243 // use master, for consistency with the getRevisionByTitle call above.
244 'use-master' => true,
245 // bypass audience checks, since we know that this is the current revision.
246 'audience' => RevisionRecord
::RAW
249 $parserOutput = $renderedRevision->getRevisionParserOutput(
250 // HTML is only needed if the output is to be placed in the parser cache
251 [ 'generate-html' => $checkCache ]
254 // If it took a long time to render, then save this back to the cache to avoid
255 // wasted CPU by other apaches or job runners. We don't want to always save to
256 // cache as this can cause high cache I/O and LRU churn when a template changes.
257 $elapsed = microtime( true ) - $start;
259 $parseThreshold = $this->params
['parseThreshold'] ?? self
::PARSE_THRESHOLD_SEC
;
261 if ( $checkCache && $elapsed >= $parseThreshold && $parserOutput->isCacheable() ) {
262 $ctime = wfTimestamp( TS_MW
, (int)$start ); // cache time
263 $services->getParserCache()->save(
264 $parserOutput, $page, $parserOptions, $ctime, $revision->getId()
267 $stats->increment( 'refreshlinks.parser_uncached' );
271 'recursive' => !empty( $this->params
['useRecursiveLinksUpdate'] ),
272 // Carry over cause so the update can do extra logging
273 'causeAction' => $this->params
['causeAction'],
274 'causeAgent' => $this->params
['causeAgent'],
276 'transactionTicket' => $ticket,
278 if ( !empty( $this->params
['triggeringUser'] ) ) {
279 $userInfo = $this->params
['triggeringUser'];
280 if ( $userInfo['userId'] ) {
281 $options['triggeringUser'] = User
::newFromId( $userInfo['userId'] );
283 // Anonymous, use the username
284 $options['triggeringUser'] = User
::newFromName( $userInfo['userName'], false );
287 $page->doSecondaryDataUpdates( $options );
289 InfoAction
::invalidateCache( $title );
291 // Commit any writes here in case this method is called in a loop.
292 // In that case, the scoped lock will fail to be acquired.
293 $lbFactory->commitAndWaitForReplication( __METHOD__
, $ticket );
298 public function getDeduplicationInfo() {
299 $info = parent
::getDeduplicationInfo();
300 unset( $info['causeAction'] );
301 unset( $info['causeAgent'] );
302 if ( is_array( $info['params'] ) ) {
303 // For per-pages jobs, the job title is that of the template that changed
304 // (or similar), so remove that since it ruins duplicate detection
305 if ( isset( $info['params']['pages'] ) ) {
306 unset( $info['namespace'] );
307 unset( $info['title'] );
314 public function workItemCount() {
315 if ( !empty( $this->params
['recursive'] ) ) {
316 return 0; // nothing actually refreshed
317 } elseif ( isset( $this->params
['pages'] ) ) {
318 return count( $this->params
['pages'] );
321 return 1; // one title